from bs4 import BeautifulSoup
import requests
import concurrent.futures
import logging

# 配置日志记录
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0'
}


class City:
    def extract_cities_from_html(self, html_code):
        cities = []
        soup = BeautifulSoup(html_code, 'html.parser')
        tr_tags = soup.find_all('tr')

        for tr_tag in tr_tags:
            td_tags = tr_tag.find_all('td')
            for td_tag in td_tags:
                city = td_tag.get_text().strip()  # 去除城市名称两端的空格
                if city and not city.isspace():  # 检查城市是否为空或只包含空格
                    cities.append(city)
        cities = [city.replace(" ", "") for city in cities]
        return cities

    def get_html(self):
        logging.info("正在爬取中国城市名单")
        html = requests.get('http://data.acmr.com.cn/member/city/city_md.asp', headers=headers)
        html.encoding = 'GBK'
        logging.info("爬取成功")
        return html.text

    def GetData(self):
        cities = City()
        html_code = cities.get_html()
        city_list = cities.extract_cities_from_html(html_code)
        logging.info("数据清洗中")
        with concurrent.futures.ThreadPoolExecutor() as executor:
            res = list(executor.map(self.process_city, city_list))

        res = [item for sublist in res for item in sublist]  # 将多个列表合并为一个列表
        res = list(set(res))  # 去除重复的城市名称
        logging.info("清洗完成")

        return res

    def process_city(self, city):
        res = []
        c = str(city).split("\n")
        for c1 in c:
            c1 = c1.split("\r")
            to_remove = ['[TOP]', '', '县级市:', '地级及以上城市:', '超大城市(市区非农人口在200万以上)',
                         '|会员中心', ' Copyright', 'Copyright', '小城市(市区非农人口在20万以下)',
                         '|统计知识', '|在线数据服务', '|联系我们',
                         '©2001-2004ACMR北京华通人商用信息有限公司版权所有', '--吉林省', '|财经资讯',
                         '|市场研究与咨询服务', ' ',
                         '特大城市(市区非农人口在100万-200万)', '2024年3月21日',
                         '京ICP证050875号 京ICP备05020597号 IP:202.100.240.250', '|企业调查与营销服务', 'English',
                         '建议使用IE5.5以上浏览器，分辨率800*600。', '|线下产品与服务']
            c1 = [char for char in c1 if char not in to_remove]
            if len(c1) != 0 and len(c1[0]) >= 6 and c1[0].find('-') == -1 and c1[0].find('您的位置') == -1 and c1[
                0].find('数据') == -1 and c1[0].find('行业') == -1:
                res.append(c1[0])
        return res
